import os
current_directory = os.getcwd()
print(current_directory)

/Users/macbookpro/hdd/MSc/Dissertation/multilabeltextclassification

import numpy
numpy.version.version

'1.22.0'

# importing libraries

from tensorflow import keras
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
#from keras.preprocessing.sequence import pad_sequences
#from keras_preprocessing.sequence import pad_sequences
#from tensorflow.keras.preprocessing.sequence import pad_sequences
from keras.utils import pad_sequences
from tensorflow.keras.preprocessing import text, sequence
from tensorflow.keras import initializers, regularizers, constraints, optimizers, layers
from tensorflow.python.keras.models import Model, Input
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import GRU, Dense, Input, LSTM, Embedding, Dropout, SpatialDropout1D, Activation, SimpleRNN
from tensorflow.keras.layers import Conv1D, Bidirectional, GlobalMaxPool1D, MaxPooling1D, BatchNormalization
from tensorflow.keras.optimizers import Adam
#from tensorflow.keras.optimizers import SGD

# For custom metrics
import keras.backend as K
from keras.utils.vis_utils import plot_model
from keras.callbacks import EarlyStopping 

import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
plt.style.use('seaborn')

import seaborn as sns
from IPython.display import Image

from tqdm import tqdm
from nltk.corpus import stopwords
from nltk.tokenize import RegexpTokenizer 
import os, re, csv, math, codecs
from nltk.tokenize import word_tokenize
import string
import gensim

sns.set_style("whitegrid")
np.random.seed(0)

2023-03-30 20:23:56.326447: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.

#df1 = pd.read_csv('/Users/macbookpro/hdd/MSc/Dissertation/multilabeltextclassification/githubissuedata2M.csv')
df = pd.read_csv('/Users/macbookpro/hdd/MSc/Dissertation/multilabeltextclassification/githubissuedata.csv')
df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 226163 entries, 0 to 226162
Data columns (total 17 columns):
 #   Column            Non-Null Count   Dtype 
---  ------            --------------   ----- 
 0   title             225152 non-null  object
 1   body              225866 non-null  object
 2   task              226163 non-null  int64 
 3   bug               226163 non-null  int64 
 4   documentation     226163 non-null  int64 
 5   duplicate         226163 non-null  int64 
 6   enhancement       226163 non-null  int64 
 7   good_first_issue  226163 non-null  int64 
 8   help_wanted       226163 non-null  int64 
 9   invalid           226163 non-null  int64 
 10  question          226163 non-null  int64 
 11  wontfix           226163 non-null  int64 
 12  gitalk            226163 non-null  int64 
 13  priority_medium   226163 non-null  int64 
 14  priority_high     226163 non-null  int64 
 15  feature_request   226163 non-null  int64 
 16  feature           226163 non-null  int64 
dtypes: int64(15), object(2)
memory usage: 29.3+ MB

df.head(10)

	title	body	task	bug	documentation	duplicate	enhancement	good_first_issue	help_wanted	invalid	question	wontfix	gitalk	priority_medium	priority_high	feature_request	feature
0	My Account Paid laptop 1440 resolution Updat...	Case:Distance between Registered email address...	0	1	0	0	0	0	0	1	0	0	0	0	0	0	0
1	How to fix sleepimpl warning when ECS credenti...	Prerequisites X Ive searched for previous sim...	0	1	0	0	0	0	0	0	0	0	0	0	0	0	0
2	Slider doesnt work on touch devices	DescriptionSlider should work dragging in tou...	0	1	0	0	0	0	0	0	0	0	0	0	0	0	0
3	add new labels	DescriptionAdd ui and logic to permanently ad...	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0
4	No lib sub folder in Boost folder	Hi I am following thishttps://www.mlpack.org/d...	0	0	0	0	0	0	1	0	1	0	0	0	0	0	0
5	Add license notice to CLI	The CLI is missing the license notice. Theres ...	0	0	0	0	0	0	0	0	0	0	0	0	0	0	1
6	Should show Powershell or AzureCLI code necess...	There is example output from Powershell and Az...	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0
7	tidboperator could not work with kubernetes 1.23	Bug ReportWhat version of Kubernetes are you ...	0	1	0	0	0	0	0	0	0	0	0	0	0	0	0
8	Match Live	x Implement game logic x Calculate results ba...	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0
9	AngularBug Make current location widget more g...	If youve never submitted an issue to the SORMA...	0	1	0	0	0	0	0	0	0	0	0	0	0	0	0

# check missing values in the dataset
print('The dataset has', df.isna().sum().sum(), 'missing values in test data.')
# check any duplicate records in the dataset
print('The dataset has', df.duplicated().sum(), 'duplicates in train data.')

The dataset has 1308 missing values in test data.
The dataset has 46411 duplicates in train data.

# remove missing values in the dataset
df.dropna(inplace= True)
# remove all duplicate records in the dataset
df.drop_duplicates(inplace= True)

# check missing values in the dataset
print('The dataset has', df.isna().sum().sum(), 'missing values in test data.')
# check any duplicate records in the dataset
print('The dataset has', df.duplicated().sum(), 'duplicates in train data.')

The dataset has 0 missing values in test data.
The dataset has 0 duplicates in train data.

# spliting dataset to train and test
from sklearn.model_selection import train_test_split
train_df, test_df = train_test_split(df, test_size=0.2, random_state=25)

test_df.shape

(35787, 17)

train_df.shape

(143145, 17)

# DATA PREPROCESSING
#Deep Neural Networks input layers make use of input variables to feed the network for training the model. But in this task (experiment), we're dealing with words text. How do we represent these words in order to feed our model?
#In our experiment, we used densed representation of those text (comments) and their semanticity together. The advantage of using this approach is the best way for fitting neural networks onto a text data (as in our case), as well as less memory usage compared to other sparse representation approaches.
#Word Embedding
#Two ways to feed embeddings to neural networks:
  #Using your own word embeddings by training
  #Using pre-trained embedding (e.g Word2vec, lad2vec, Glove etc)

#Convert text to vectors using keras preprocessing library tools


X_train = train_df["body"].values
X_test  = test_df["body"].values

y_train = train_df[["task","bug","documentation","duplicate","enhancement","good_first_issue","help_wanted","invalid","question","wontfix","gitalk","priority_medium","priority_high","feature_request","feature"]].values
y_test  = test_df[["task","bug","documentation","duplicate","enhancement","good_first_issue","help_wanted","invalid","question","wontfix","gitalk","priority_medium","priority_high","feature_request","feature"]].values

#For the first embedding, we used keras preprocessing (Text Preprocessing) libraries. 
#This class allows to vectorize a text corpus, by turning each text into either a sequence of integers 
#(each integer being the index of a token in a dictionary) or into a vector where the coefficient for 
#each token could be binary, based on word count, based on tf-idf

num_words = 20000 #Max. words to use per issue label
max_features = 200000 #Max. number of unique words in embeddinbg vector
max_len = 500 #Max. number of words per issue to be use
embedding_dims = 128 #embedding vector output dimension 
num_epochs = 25 # (before 5)number of epochs (number of times that the model is exposed to the training dataset)
val_split = 0.1
batch_size2 = 256 #(before 32)The **batch size** is the number of training examples in one forward/backward pass.
                  # In general, larger batch sizes result in faster progress in training, but don't always converge as quickly. 
                  #Smaller batch sizes train slower, but can converge faster. And the higher the batch size, the more memory space you’ll need.

X_train[1]

'Tidying up the code to remove redundant processing'

#Issue body Tokenization
tokenizer = tokenizer = Tokenizer(num_words)
tokenizer.fit_on_texts(list(X_train))

#Convert tokenized issue body text to sequnces
X_train = tokenizer.texts_to_sequences(X_train)
X_test = tokenizer.texts_to_sequences(X_test)
 
# padding the sequences
X_train = pad_sequences(X_train, max_len)
X_test  = pad_sequences(X_test,  max_len)

print('X_train shape:', X_train.shape)
print('X_test shape: ', X_test.shape)

X_train shape: (143145, 500)
X_test shape:  (35787, 500)

#We used early callback functionality that allows you to specify the performance measure to monitor, the trigger, and once triggered. It will stop the training process.
early = EarlyStopping(monitor="val_loss", mode="min", patience=4)

#Writing functions for Precision, Recall, F1-Measure, AUC, mean etc evaluaiton metrics to evaluate the model

#Import necessary libraries
4# demonstration of calculating metrics for a neural network model using sklearn
from sklearn.datasets import make_circles
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score
from sklearn.metrics import cohen_kappa_score
from sklearn.metrics import roc_auc_score
from sklearn.metrics import confusion_matrix


def precision(y_true, y_pred):
    #Calculating precision, a metric for multi-label classification of how many selected items are relevant.
    
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
    precision = true_positives / (predicted_positives + K.epsilon())
    return precision


def recall(y_true, y_pred):
    #Calculating recall, a metric for multi-label classification of how many relevant items are selected.
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
    recall = true_positives / (possible_positives + K.epsilon())
    return recall

  #Customized the evaluation to analyse the model in terms of accuracy and mean value accuracy
def mean_pred(y_true, y_pred):
    return K.mean(y_pred)

def fbeta_score(y_true, y_pred, beta=1):
    '''Calculates the F score, the weighted harmonic mean of precision and recall.
    This is useful for multi-label classification, where input samples can be
    classified as sets of labels. By only using accuracy (precision) a model
    would achieve a perfect score by simply assigning every class to every
    input. In order to avoid this, a metric should penalize incorrect class
    assignments as well (recall). The F-beta score (ranged from 0.0 to 1.0)
    computes this, as a weighted mean of the proportion of correct class
    assignments vs. the proportion of incorrect class assignments.
    With beta = 1, this is equivalent to a F-measure. With beta < 1, assigning
    correct classes becomes more important, and with beta > 1 the metric is
    instead weighted towards penalizing incorrect class assignments.
    '''
    if beta < 0:
        raise ValueError('The lowest choosable beta is zero (only precision).')
        
    # If there are no true positives, fix the F score at 0 like sklearn.
    if K.sum(K.round(K.clip(y_true, 0, 1))) == 0:
        return 0.0

    p = precision(y_true, y_pred)
    r = recall(y_true, y_pred)
    bb = beta ** 2
    fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())
    return fbeta_score

def auroc(y_true, y_pred):
    auc = tf.keras.metrics.AUC(y_true, y_pred)[1]
    #auc = tf.metrics.auc(y_true, y_pred)[1]
    K.get_session().run(tf.local_variables_initializer())
    return auc

def fmeasure(y_true, y_pred):
    #Calculates the f-measure, the harmonic mean of precision and recall.
    return fbeta_score(y_true, y_pred, beta=1)


fscore = f1score = fmeasure

#1) Neural Network (NN)
nn_model = Sequential()
embedding_layer = Embedding(input_dim=max_features,output_dim=embedding_dims,input_length=max_len)
nn_model.add(embedding_layer)
nn_model.add(GlobalMaxPool1D())
nn_model.add(Dense(50,activation='relu'))
nn_model.add(Dropout(0.1))
nn_model.add(Dense(15,activation='sigmoid'))

opt = Adam(learning_rate=0.1)
nn_model.compile(loss='binary_crossentropy',optimizer=opt,metrics=['accuracy', mean_pred, fmeasure, precision, recall])

#input_shape = X_train.shape  
#nn_model.build(input_shape) 
print(nn_model.summary())

Model: "sequential"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding (Embedding)       (None, 500, 128)          25600000  
                                                                 
 global_max_pooling1d (Globa  (None, 128)              0         
 lMaxPooling1D)                                                  
                                                                 
 dense (Dense)               (None, 50)                6450      
                                                                 
 dropout (Dropout)           (None, 50)                0         
                                                                 
 dense_1 (Dense)             (None, 15)                765       
                                                                 
=================================================================
Total params: 25,607,215
Trainable params: 25,607,215
Non-trainable params: 0
_________________________________________________________________
None

nn_model_fit = nn_model.fit(X_train, y_train, batch_size=batch_size2, epochs=num_epochs, validation_split=val_split, callbacks=[early])

Epoch 1/25
504/504 [==============================] - 151s 297ms/step - loss: 0.1794 - accuracy: 0.5938 - mean_pred: 0.0841 - fmeasure: 0.4580 - precision: 0.7812 - recall: 0.3326 - val_loss: 0.1710 - val_accuracy: 0.6105 - val_mean_pred: 0.0818 - val_fmeasure: 0.4226 - val_precision: 0.8902 - val_recall: 0.2775
Epoch 2/25
504/504 [==============================] - 149s 296ms/step - loss: 0.1741 - accuracy: 0.5950 - mean_pred: 0.0809 - fmeasure: 0.4251 - precision: 0.8658 - recall: 0.2838 - val_loss: 0.1696 - val_accuracy: 0.6208 - val_mean_pred: 0.0797 - val_fmeasure: 0.4421 - val_precision: 0.8657 - val_recall: 0.2975
Epoch 3/25
504/504 [==============================] - 149s 295ms/step - loss: 0.1730 - accuracy: 0.5944 - mean_pred: 0.0808 - fmeasure: 0.4230 - precision: 0.8838 - recall: 0.2808 - val_loss: 0.1774 - val_accuracy: 0.6137 - val_mean_pred: 0.0783 - val_fmeasure: 0.4418 - val_precision: 0.8720 - val_recall: 0.2964
Epoch 4/25
504/504 [==============================] - 149s 295ms/step - loss: 0.1719 - accuracy: 0.5980 - mean_pred: 0.0808 - fmeasure: 0.4264 - precision: 0.8980 - recall: 0.2816 - val_loss: 0.1792 - val_accuracy: 0.5572 - val_mean_pred: 0.0804 - val_fmeasure: 0.3364 - val_precision: 0.9343 - val_recall: 0.2056
Epoch 5/25
504/504 [==============================] - 149s 296ms/step - loss: 0.1692 - accuracy: 0.6052 - mean_pred: 0.0808 - fmeasure: 0.4356 - precision: 0.9053 - recall: 0.2893 - val_loss: 0.1762 - val_accuracy: 0.6245 - val_mean_pred: 0.0850 - val_fmeasure: 0.4726 - val_precision: 0.8244 - val_recall: 0.3319
Epoch 6/25
504/504 [==============================] - 150s 298ms/step - loss: 0.1757 - accuracy: 0.5725 - mean_pred: 0.0809 - fmeasure: 0.3810 - precision: 0.8994 - recall: 0.2523 - val_loss: 0.1974 - val_accuracy: 0.4872 - val_mean_pred: 0.0812 - val_fmeasure: 0.0303 - val_precision: 0.9721 - val_recall: 0.0154

#from tensorflow.keras.utils import plot_model

#plot the nn_model architecture

#plot_model(nn_model, to_file='nn_model_plot.png', show_shapes=True, show_layer_names=True)
#Image(retina=True, filename='nn_model_plot.png')

# Plot training & validation accuracy values
plt.plot(nn_model_fit.history['accuracy'])
plt.plot(nn_model_fit.history['val_accuracy'])
plt.title('Neural Network (NN) Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(nn_model_fit.history['loss'])
plt.plot(nn_model_fit.history['val_loss'])
plt.title('Neural Network (NN) Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()

#2) Convolutional Neural Network (CNN)

CNN_model = Sequential([
    Embedding(input_dim=max_features, input_length=max_len, output_dim=embedding_dims),
    SpatialDropout1D(0.5),
    # ... 100 filters with a kernel size of 4 so that each convolution will consider a window of 4 word embeddings
    Conv1D(filters=100, kernel_size=4, padding='same', activation='relu'),
    #**batch normalization layer** normalizes the activations of the previous layer at each batch, 
    #i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1. 
    #It will be added after the activation function between a convolutional and a max-pooling layer.
    BatchNormalization(),
    Dropout(0.5),
    GlobalMaxPool1D(),
    Dense(50, activation = 'relu'),
    Dense(15, activation = 'sigmoid')
])

#Customized the evaluation to analyse the model in terms of accuracy and mean value accuracy
def mean_pred(y_true, y_pred):
    return K.mean(y_pred)

CNN_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])

CNN_model.summary()

Model: "sequential_1"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding_1 (Embedding)     (None, 500, 128)          25600000  
                                                                 
 spatial_dropout1d (SpatialD  (None, 500, 128)         0         
 ropout1D)                                                       
                                                                 
 conv1d (Conv1D)             (None, 500, 100)          51300     
                                                                 
 batch_normalization (BatchN  (None, 500, 100)         400       
 ormalization)                                                   
                                                                 
 dropout_1 (Dropout)         (None, 500, 100)          0         
                                                                 
 global_max_pooling1d_1 (Glo  (None, 100)              0         
 balMaxPooling1D)                                                
                                                                 
 dense_2 (Dense)             (None, 50)                5050      
                                                                 
 dense_3 (Dense)             (None, 15)                765       
                                                                 
=================================================================
Total params: 25,657,515
Trainable params: 25,657,315
Non-trainable params: 200
_________________________________________________________________

plot_model(nn_model, to_file='nn_model_plot.png', show_shapes=True, show_layer_names=True)
#Image(retina=True, filename='nn_model_plot.png')

You must install pydot (`pip install pydot`) and install graphviz (see instructions at https://graphviz.gitlab.io/download/) for plot_model to work.

CNN_model_fit = CNN_model.fit(X_train, y_train, batch_size=batch_size2, epochs=num_epochs, validation_split=val_split, callbacks=[early])

Epoch 1/25
504/504 [==============================] - 378s 748ms/step - loss: 0.1761 - accuracy: 0.6189 - mean_pred: 0.0842 - fmeasure: 0.5367 - precision: 0.7220 - recall: 0.4297 - val_loss: 0.1749 - val_accuracy: 0.6611 - val_mean_pred: 0.1198 - val_fmeasure: 0.5880 - val_precision: 0.7610 - val_recall: 0.4796
Epoch 2/25
504/504 [==============================] - 375s 745ms/step - loss: 0.1511 - accuracy: 0.6698 - mean_pred: 0.0817 - fmeasure: 0.6062 - precision: 0.7593 - recall: 0.5054 - val_loss: 0.1643 - val_accuracy: 0.6699 - val_mean_pred: 0.1101 - val_fmeasure: 0.5760 - val_precision: 0.7976 - val_recall: 0.4511
Epoch 3/25
504/504 [==============================] - 372s 738ms/step - loss: 0.1436 - accuracy: 0.6842 - mean_pred: 0.0814 - fmeasure: 0.6284 - precision: 0.7714 - recall: 0.5308 - val_loss: 0.1709 - val_accuracy: 0.6740 - val_mean_pred: 0.1181 - val_fmeasure: 0.5766 - val_precision: 0.8017 - val_recall: 0.4506
Epoch 4/25
504/504 [==============================] - 384s 762ms/step - loss: 0.1386 - accuracy: 0.6945 - mean_pred: 0.0812 - fmeasure: 0.6414 - precision: 0.7794 - recall: 0.5456 - val_loss: 0.1611 - val_accuracy: 0.6808 - val_mean_pred: 0.1089 - val_fmeasure: 0.5987 - val_precision: 0.7907 - val_recall: 0.4822
Epoch 5/25
504/504 [==============================] - 381s 755ms/step - loss: 0.1351 - accuracy: 0.7033 - mean_pred: 0.0811 - fmeasure: 0.6520 - precision: 0.7885 - recall: 0.5564 - val_loss: 0.1566 - val_accuracy: 0.6778 - val_mean_pred: 0.1009 - val_fmeasure: 0.5904 - val_precision: 0.8048 - val_recall: 0.4667
Epoch 6/25
504/504 [==============================] - 381s 756ms/step - loss: 0.1325 - accuracy: 0.7090 - mean_pred: 0.0810 - fmeasure: 0.6589 - precision: 0.7919 - recall: 0.5648 - val_loss: 0.1549 - val_accuracy: 0.6789 - val_mean_pred: 0.0961 - val_fmeasure: 0.6000 - val_precision: 0.7949 - val_recall: 0.4823
Epoch 7/25
504/504 [==============================] - 380s 754ms/step - loss: 0.1296 - accuracy: 0.7159 - mean_pred: 0.0810 - fmeasure: 0.6664 - precision: 0.7963 - recall: 0.5736 - val_loss: 0.1550 - val_accuracy: 0.6755 - val_mean_pred: 0.0881 - val_fmeasure: 0.6147 - val_precision: 0.7648 - val_recall: 0.5143
Epoch 8/25
504/504 [==============================] - 379s 752ms/step - loss: 0.1277 - accuracy: 0.7191 - mean_pred: 0.0810 - fmeasure: 0.6716 - precision: 0.8006 - recall: 0.5790 - val_loss: 0.1644 - val_accuracy: 0.6651 - val_mean_pred: 0.0914 - val_fmeasure: 0.5824 - val_precision: 0.7591 - val_recall: 0.4729
Epoch 9/25
504/504 [==============================] - 380s 754ms/step - loss: 0.1255 - accuracy: 0.7245 - mean_pred: 0.0809 - fmeasure: 0.6783 - precision: 0.8044 - recall: 0.5869 - val_loss: 0.1595 - val_accuracy: 0.6737 - val_mean_pred: 0.0865 - val_fmeasure: 0.5956 - val_precision: 0.7790 - val_recall: 0.4826
Epoch 10/25
504/504 [==============================] - 379s 751ms/step - loss: 0.1239 - accuracy: 0.7286 - mean_pred: 0.0809 - fmeasure: 0.6819 - precision: 0.8059 - recall: 0.5915 - val_loss: 0.1609 - val_accuracy: 0.6692 - val_mean_pred: 0.0860 - val_fmeasure: 0.6098 - val_precision: 0.7481 - val_recall: 0.5151

CNN_train_score = CNN_model.evaluate(X_train, y_train, batch_size = batch_size2, verbose = 1)
print('Train loss:', CNN_train_score[0])
print('Train accuracy:', CNN_train_score[1])

560/560 [==============================] - 48s 85ms/step - loss: 0.1398 - accuracy: 0.7413 - mean_pred: 0.0859 - fmeasure: 0.6834 - precision: 0.8278 - recall: 0.5823
Train loss: 0.13975094258785248
Train accuracy: 0.7412553429603577

CNN_test_score = CNN_model.evaluate(X_test, y_test, batch_size=batch_size2, verbose=1)
print('Test Loss:', CNN_test_score[0])
print('Test Accuracy:', CNN_test_score[1])

140/140 [==============================] - 12s 83ms/step - loss: 0.1603 - accuracy: 0.6792 - mean_pred: 0.0860 - fmeasure: 0.6204 - precision: 0.7578 - recall: 0.5255
Test Loss: 0.16033262014389038
Test Accuracy: 0.6792410612106323

# Plot training & validation accuracy values
plt.plot(CNN_model_fit.history['accuracy'])
plt.plot(CNN_model_fit.history['val_accuracy'])
plt.title('Convolutional Neural Network (CNN) Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(CNN_model_fit.history['loss'])
plt.plot(CNN_model_fit.history['val_loss'])
plt.title('Convolutional Neural Network (CNN) Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()

#3) Recurrent Neural Networks (RNN)

RNN_model = Sequential([
    Embedding(input_dim=max_features, input_length=max_len, output_dim=embedding_dims),
    #Bidirectional layer will enable our model to predict a missing word in a sequence, 
    #So, using this feature will enable the model to look at the context on both the left and the right.
    SimpleRNN(128),
    #Bidirectional(LSTM(25, return_sequences=True)),
    #**batch normalization layer** normalizes the activations of the previous layer at each batch, 
    #i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1. 
    Dense(15, activation = 'sigmoid')
])

#Customized the evaluation to analyse the model in terms of accuracy and mean value accuracy
def mean_pred(y_true, y_pred):
    return K.mean(y_pred)

RNN_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])

RNN_model.summary()

Model: "sequential_2"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding_2 (Embedding)     (None, 500, 128)          25600000  
                                                                 
 simple_rnn (SimpleRNN)      (None, 128)               32896     
                                                                 
 dense_4 (Dense)             (None, 15)                1935      
                                                                 
=================================================================
Total params: 25,634,831
Trainable params: 25,634,831
Non-trainable params: 0
_________________________________________________________________

RNN_model_fit = RNN_model.fit(X_train, y_train, batch_size=batch_size2, epochs=num_epochs, validation_split=val_split, callbacks=[early])

Epoch 1/25
504/504 [==============================] - 369s 730ms/step - loss: 0.2569 - accuracy: 0.4592 - mean_pred: 0.1296 - fmeasure: 0.2874 - precision: 0.4462 - recall: 0.2408 - val_loss: 0.2080 - val_accuracy: 0.4839 - val_mean_pred: 0.0961 - val_fmeasure: 0.3673 - val_precision: 0.5144 - val_recall: 0.2864
Epoch 2/25
504/504 [==============================] - 367s 727ms/step - loss: 0.2045 - accuracy: 0.4995 - mean_pred: 0.0952 - fmeasure: 0.3451 - precision: 0.5948 - recall: 0.2453 - val_loss: 0.2057 - val_accuracy: 0.4842 - val_mean_pred: 0.0948 - val_fmeasure: 0.3166 - val_precision: 0.5675 - val_recall: 0.2200
Epoch 3/25
504/504 [==============================] - 369s 733ms/step - loss: 0.1970 - accuracy: 0.4950 - mean_pred: 0.0868 - fmeasure: 0.3279 - precision: 0.6161 - recall: 0.2242 - val_loss: 0.1954 - val_accuracy: 0.4893 - val_mean_pred: 0.0852 - val_fmeasure: 0.3286 - val_precision: 0.6093 - val_recall: 0.2253
Epoch 4/25
504/504 [==============================] - 2201s 4s/step - loss: 0.1915 - accuracy: 0.5044 - mean_pred: 0.0829 - fmeasure: 0.3531 - precision: 0.6414 - recall: 0.2443 - val_loss: 0.1946 - val_accuracy: 0.4772 - val_mean_pred: 0.0824 - val_fmeasure: 0.3510 - val_precision: 0.5886 - val_recall: 0.2503
Epoch 5/25
504/504 [==============================] - 374s 742ms/step - loss: 0.1912 - accuracy: 0.5106 - mean_pred: 0.0823 - fmeasure: 0.3655 - precision: 0.6373 - recall: 0.2570 - val_loss: 0.1949 - val_accuracy: 0.4844 - val_mean_pred: 0.0829 - val_fmeasure: 0.3448 - val_precision: 0.5966 - val_recall: 0.2428
Epoch 6/25
504/504 [==============================] - 382s 758ms/step - loss: 0.1919 - accuracy: 0.5089 - mean_pred: 0.0820 - fmeasure: 0.3679 - precision: 0.6309 - recall: 0.2603 - val_loss: 0.1949 - val_accuracy: 0.4877 - val_mean_pred: 0.0822 - val_fmeasure: 0.3484 - val_precision: 0.5931 - val_recall: 0.2470
Epoch 7/25
504/504 [==============================] - 441s 875ms/step - loss: 0.1901 - accuracy: 0.5162 - mean_pred: 0.0815 - fmeasure: 0.3767 - precision: 0.6411 - recall: 0.2674 - val_loss: 0.1938 - val_accuracy: 0.4882 - val_mean_pred: 0.0814 - val_fmeasure: 0.3598 - val_precision: 0.5954 - val_recall: 0.2581
Epoch 8/25
504/504 [==============================] - 403s 800ms/step - loss: 0.1886 - accuracy: 0.5215 - mean_pred: 0.0811 - fmeasure: 0.3938 - precision: 0.6494 - recall: 0.2832 - val_loss: 0.1939 - val_accuracy: 0.4912 - val_mean_pred: 0.0824 - val_fmeasure: 0.3667 - val_precision: 0.5980 - val_recall: 0.2648
Epoch 9/25
504/504 [==============================] - 416s 825ms/step - loss: 0.1893 - accuracy: 0.5204 - mean_pred: 0.0812 - fmeasure: 0.3954 - precision: 0.6414 - recall: 0.2871 - val_loss: 0.1928 - val_accuracy: 0.4873 - val_mean_pred: 0.0879 - val_fmeasure: 0.3843 - val_precision: 0.6027 - val_recall: 0.2823
Epoch 10/25
504/504 [==============================] - 421s 835ms/step - loss: 0.1870 - accuracy: 0.5269 - mean_pred: 0.0810 - fmeasure: 0.4063 - precision: 0.6512 - recall: 0.2968 - val_loss: 0.1906 - val_accuracy: 0.5039 - val_mean_pred: 0.0781 - val_fmeasure: 0.3778 - val_precision: 0.6198 - val_recall: 0.2721
Epoch 11/25
504/504 [==============================] - 417s 827ms/step - loss: 0.1882 - accuracy: 0.5276 - mean_pred: 0.0807 - fmeasure: 0.4087 - precision: 0.6432 - recall: 0.3009 - val_loss: 0.1962 - val_accuracy: 0.4931 - val_mean_pred: 0.0792 - val_fmeasure: 0.3830 - val_precision: 0.5833 - val_recall: 0.2855
Epoch 12/25
504/504 [==============================] - 452s 896ms/step - loss: 0.1895 - accuracy: 0.5211 - mean_pred: 0.0809 - fmeasure: 0.3999 - precision: 0.6321 - recall: 0.2932 - val_loss: 0.1934 - val_accuracy: 0.4939 - val_mean_pred: 0.0800 - val_fmeasure: 0.3758 - val_precision: 0.6065 - val_recall: 0.2725
Epoch 13/25
504/504 [==============================] - 420s 834ms/step - loss: 0.1871 - accuracy: 0.5271 - mean_pred: 0.0809 - fmeasure: 0.4129 - precision: 0.6445 - recall: 0.3046 - val_loss: 0.1944 - val_accuracy: 0.4948 - val_mean_pred: 0.0815 - val_fmeasure: 0.4014 - val_precision: 0.5869 - val_recall: 0.3053
Epoch 14/25
504/504 [==============================] - 428s 849ms/step - loss: 0.1856 - accuracy: 0.5344 - mean_pred: 0.0809 - fmeasure: 0.4210 - precision: 0.6532 - recall: 0.3115 - val_loss: 0.1927 - val_accuracy: 0.5000 - val_mean_pred: 0.0819 - val_fmeasure: 0.3987 - val_precision: 0.6003 - val_recall: 0.2987

# Plot training & validation accuracy values
plt.plot(RNN_model_fit.history['accuracy'])
plt.plot(RNN_model_fit.history['val_accuracy'])
plt.title('Recurrent Neural Networks (RNNs) Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(RNN_model_fit.history['loss'])
plt.plot(RNN_model_fit.history['val_loss'])
plt.title('Recurrent Neural Networks (RNNs) Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()

#4) LSTM

LSTM_model = Sequential([
    Embedding(input_dim=max_features, input_length=max_len, output_dim=embedding_dims),
    SpatialDropout1D(0.5),
    LSTM(64, dropout=0.2, recurrent_dropout=0.2),
    #**batch normalization layer** normalizes the activations of the previous layer at each batch, 
    #i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1. 
    #It will be added after the activation function between a convolutional and a max-pooling layer.
    BatchNormalization(),
    Dropout(0.5),
    Dense(50, activation = 'relu'),
    Dense(15, activation = 'sigmoid')
])

LSTM_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])

LSTM_model.summary()

Model: "sequential_3"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding_3 (Embedding)     (None, 500, 128)          25600000  
                                                                 
 spatial_dropout1d_1 (Spatia  (None, 500, 128)         0         
 lDropout1D)                                                     
                                                                 
 lstm (LSTM)                 (None, 64)                49408     
                                                                 
 batch_normalization_1 (Batc  (None, 64)               256       
 hNormalization)                                                 
                                                                 
 dropout_2 (Dropout)         (None, 64)                0         
                                                                 
 dense_5 (Dense)             (None, 50)                3250      
                                                                 
 dense_6 (Dense)             (None, 15)                765       
                                                                 
=================================================================
Total params: 25,653,679
Trainable params: 25,653,551
Non-trainable params: 128
_________________________________________________________________

LSTM_model_fit = LSTM_model.fit(X_train, y_train, batch_size=batch_size2, epochs=num_epochs, validation_split=val_split, callbacks=[early])

Epoch 1/25
504/504 [==============================] - 951s 2s/step - loss: 0.1802 - accuracy: 0.5998 - mean_pred: 0.0857 - fmeasure: 0.5042 - precision: 0.7102 - recall: 0.3960 - val_loss: 0.1608 - val_accuracy: 0.6407 - val_mean_pred: 0.0874 - val_fmeasure: 0.5898 - val_precision: 0.7075 - val_recall: 0.5060
Epoch 2/25
504/504 [==============================] - 939s 2s/step - loss: 0.1572 - accuracy: 0.6565 - mean_pred: 0.0811 - fmeasure: 0.5839 - precision: 0.7484 - recall: 0.4798 - val_loss: 0.1523 - val_accuracy: 0.6669 - val_mean_pred: 0.0831 - val_fmeasure: 0.6195 - val_precision: 0.7231 - val_recall: 0.5421
Epoch 3/25
504/504 [==============================] - 950s 2s/step - loss: 0.1497 - accuracy: 0.6755 - mean_pred: 0.0810 - fmeasure: 0.6113 - precision: 0.7628 - recall: 0.5107 - val_loss: 0.1504 - val_accuracy: 0.6710 - val_mean_pred: 0.0814 - val_fmeasure: 0.6243 - val_precision: 0.7300 - val_recall: 0.5457
Epoch 4/25
504/504 [==============================] - 885s 2s/step - loss: 0.1462 - accuracy: 0.6840 - mean_pred: 0.0810 - fmeasure: 0.6243 - precision: 0.7681 - recall: 0.5265 - val_loss: 0.1498 - val_accuracy: 0.6713 - val_mean_pred: 0.0826 - val_fmeasure: 0.6223 - val_precision: 0.7451 - val_recall: 0.5346
Epoch 5/25
504/504 [==============================] - 865s 2s/step - loss: 0.1444 - accuracy: 0.6909 - mean_pred: 0.0810 - fmeasure: 0.6312 - precision: 0.7736 - recall: 0.5336 - val_loss: 0.1487 - val_accuracy: 0.6724 - val_mean_pred: 0.0814 - val_fmeasure: 0.6204 - val_precision: 0.7479 - val_recall: 0.5303
Epoch 6/25
504/504 [==============================] - 889s 2s/step - loss: 0.1437 - accuracy: 0.6916 - mean_pred: 0.0810 - fmeasure: 0.6334 - precision: 0.7761 - recall: 0.5355 - val_loss: 0.1507 - val_accuracy: 0.6789 - val_mean_pred: 0.0813 - val_fmeasure: 0.6319 - val_precision: 0.7286 - val_recall: 0.5582
Epoch 7/25
504/504 [==============================] - 961s 2s/step - loss: 0.1431 - accuracy: 0.6935 - mean_pred: 0.0810 - fmeasure: 0.6359 - precision: 0.7788 - recall: 0.5378 - val_loss: 0.1494 - val_accuracy: 0.6764 - val_mean_pred: 0.0802 - val_fmeasure: 0.6300 - val_precision: 0.7386 - val_recall: 0.5494
Epoch 8/25
504/504 [==============================] - 962s 2s/step - loss: 0.1425 - accuracy: 0.6935 - mean_pred: 0.0810 - fmeasure: 0.6356 - precision: 0.7799 - recall: 0.5368 - val_loss: 0.1491 - val_accuracy: 0.6768 - val_mean_pred: 0.0834 - val_fmeasure: 0.6266 - val_precision: 0.7378 - val_recall: 0.5449
Epoch 9/25
504/504 [==============================] - 1016s 2s/step - loss: 0.1425 - accuracy: 0.6940 - mean_pred: 0.0810 - fmeasure: 0.6353 - precision: 0.7796 - recall: 0.5366 - val_loss: 0.1504 - val_accuracy: 0.6673 - val_mean_pred: 0.0851 - val_fmeasure: 0.6285 - val_precision: 0.7335 - val_recall: 0.5502

# Plot training & validation accuracy values
plt.plot(LSTM_model_fit.history['accuracy'])
plt.plot(LSTM_model_fit.history['val_accuracy'])
plt.title('LSTM Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(LSTM_model_fit.history['loss'])
plt.plot(LSTM_model_fit.history['val_loss'])
plt.title('LSTM Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()

#5) BI-LSTM
BI_LSTM_model = Sequential([
    Embedding(input_dim=max_features, input_length=max_len, output_dim=embedding_dims),
    SpatialDropout1D(0.5),
    #Bidirectional layer will enable our model to predict a missing word in a sequence, 
    #So, using this feature will enable the model to look at the context on both the left and the right.
    Bidirectional(LSTM(25, return_sequences=True)),
    #**batch normalization layer** normalizes the activations of the previous layer at each batch, 
    #i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1. 
    BatchNormalization(),
    Dropout(0.5),
    GlobalMaxPool1D(),
    Dense(50, activation = 'relu'),
    Dense(15, activation = 'sigmoid')
])

2023-03-31 02:08:32.627716: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-03-31 02:08:32.629920: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-03-31 02:08:32.631789: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-03-31 02:08:32.801522: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
	 [[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-03-31 02:08:32.857269: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-03-31 02:08:32.858985: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-03-31 02:08:32.861112: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]

BI_LSTM_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])

BI_LSTM_model.summary()

Model: "sequential_4"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding_4 (Embedding)     (None, 500, 128)          25600000  
                                                                 
 spatial_dropout1d_2 (Spatia  (None, 500, 128)         0         
 lDropout1D)                                                     
                                                                 
 bidirectional (Bidirectiona  (None, 500, 50)          30800     
 l)                                                              
                                                                 
 batch_normalization_2 (Batc  (None, 500, 50)          200       
 hNormalization)                                                 
                                                                 
 dropout_3 (Dropout)         (None, 500, 50)           0         
                                                                 
 global_max_pooling1d_2 (Glo  (None, 50)               0         
 balMaxPooling1D)                                                
                                                                 
 dense_7 (Dense)             (None, 50)                2550      
                                                                 
 dense_8 (Dense)             (None, 15)                765       
                                                                 
=================================================================
Total params: 25,634,315
Trainable params: 25,634,215
Non-trainable params: 100
_________________________________________________________________

BI_LSTM_model_fit = BI_LSTM_model.fit(X_train, y_train, batch_size=batch_size2, epochs=num_epochs, validation_split=val_split, callbacks=[early])

Epoch 1/25

2023-03-31 02:08:33.278574: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-03-31 02:08:33.280823: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-03-31 02:08:33.282445: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-03-31 02:08:33.453860: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
	 [[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-03-31 02:08:33.511289: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-03-31 02:08:33.513691: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-03-31 02:08:33.515238: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-03-31 02:08:34.206337: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
	 [[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-03-31 02:08:34.950324: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-03-31 02:08:34.952772: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-03-31 02:08:34.954594: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-03-31 02:08:35.129403: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
	 [[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-03-31 02:08:35.189132: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-03-31 02:08:35.191367: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-03-31 02:08:35.193125: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-03-31 02:08:35.921626: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
	 [[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]

504/504 [==============================] - ETA: 0s - loss: 0.1681 - accuracy: 0.6316 - mean_pred: 0.0829 - fmeasure: 0.5514 - precision: 0.7355 - recall: 0.4453

2023-03-31 02:19:53.758388: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-03-31 02:19:53.760774: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-03-31 02:19:53.762359: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-03-31 02:19:53.929723: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
	 [[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-03-31 02:19:53.984536: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-03-31 02:19:53.986778: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-03-31 02:19:53.988763: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]

504/504 [==============================] - 702s 1s/step - loss: 0.1681 - accuracy: 0.6316 - mean_pred: 0.0829 - fmeasure: 0.5514 - precision: 0.7355 - recall: 0.4453 - val_loss: 0.2095 - val_accuracy: 0.6666 - val_mean_pred: 0.1593 - val_fmeasure: 0.5760 - val_precision: 0.7907 - val_recall: 0.4533
Epoch 2/25
504/504 [==============================] - 748s 1s/step - loss: 0.1470 - accuracy: 0.6792 - mean_pred: 0.0812 - fmeasure: 0.6176 - precision: 0.7660 - recall: 0.5182 - val_loss: 0.1770 - val_accuracy: 0.6795 - val_mean_pred: 0.1322 - val_fmeasure: 0.6218 - val_precision: 0.7641 - val_recall: 0.5245
Epoch 3/25
504/504 [==============================] - 727s 1s/step - loss: 0.1403 - accuracy: 0.6958 - mean_pred: 0.0812 - fmeasure: 0.6399 - precision: 0.7796 - recall: 0.5434 - val_loss: 0.1660 - val_accuracy: 0.6855 - val_mean_pred: 0.1188 - val_fmeasure: 0.6089 - val_precision: 0.7956 - val_recall: 0.4935
Epoch 4/25
504/504 [==============================] - 616s 1s/step - loss: 0.1369 - accuracy: 0.7042 - mean_pred: 0.0811 - fmeasure: 0.6508 - precision: 0.7877 - recall: 0.5550 - val_loss: 0.1672 - val_accuracy: 0.6827 - val_mean_pred: 0.1192 - val_fmeasure: 0.6138 - val_precision: 0.7879 - val_recall: 0.5030
Epoch 5/25
504/504 [==============================] - 625s 1s/step - loss: 0.1353 - accuracy: 0.7070 - mean_pred: 0.0811 - fmeasure: 0.6538 - precision: 0.7905 - recall: 0.5579 - val_loss: 0.1631 - val_accuracy: 0.6803 - val_mean_pred: 0.1150 - val_fmeasure: 0.6144 - val_precision: 0.7847 - val_recall: 0.5052
Epoch 6/25
504/504 [==============================] - 640s 1s/step - loss: 0.1343 - accuracy: 0.7096 - mean_pred: 0.0810 - fmeasure: 0.6565 - precision: 0.7935 - recall: 0.5603 - val_loss: 0.1590 - val_accuracy: 0.6849 - val_mean_pred: 0.1082 - val_fmeasure: 0.6051 - val_precision: 0.7980 - val_recall: 0.4876
Epoch 7/25
504/504 [==============================] - 729s 1s/step - loss: 0.1337 - accuracy: 0.7098 - mean_pred: 0.0810 - fmeasure: 0.6578 - precision: 0.7935 - recall: 0.5621 - val_loss: 0.1546 - val_accuracy: 0.6809 - val_mean_pred: 0.1021 - val_fmeasure: 0.6144 - val_precision: 0.7797 - val_recall: 0.5074
Epoch 8/25
504/504 [==============================] - 680s 1s/step - loss: 0.1335 - accuracy: 0.7101 - mean_pred: 0.0810 - fmeasure: 0.6588 - precision: 0.7956 - recall: 0.5627 - val_loss: 0.1555 - val_accuracy: 0.6719 - val_mean_pred: 0.0997 - val_fmeasure: 0.6007 - val_precision: 0.7791 - val_recall: 0.4892
Epoch 9/25
504/504 [==============================] - 697s 1s/step - loss: 0.1330 - accuracy: 0.7128 - mean_pred: 0.0809 - fmeasure: 0.6599 - precision: 0.7944 - recall: 0.5649 - val_loss: 0.1570 - val_accuracy: 0.6660 - val_mean_pred: 0.1007 - val_fmeasure: 0.5984 - val_precision: 0.7798 - val_recall: 0.4858
Epoch 10/25
504/504 [==============================] - 686s 1s/step - loss: 0.1332 - accuracy: 0.7114 - mean_pred: 0.0810 - fmeasure: 0.6602 - precision: 0.7946 - recall: 0.5651 - val_loss: 0.1512 - val_accuracy: 0.6775 - val_mean_pred: 0.0944 - val_fmeasure: 0.6075 - val_precision: 0.7775 - val_recall: 0.4989
Epoch 11/25
504/504 [==============================] - 715s 1s/step - loss: 0.1335 - accuracy: 0.7093 - mean_pred: 0.0810 - fmeasure: 0.6575 - precision: 0.7937 - recall: 0.5616 - val_loss: 0.1523 - val_accuracy: 0.6716 - val_mean_pred: 0.0931 - val_fmeasure: 0.6047 - val_precision: 0.7702 - val_recall: 0.4981
Epoch 12/25
504/504 [==============================] - 674s 1s/step - loss: 0.1337 - accuracy: 0.7103 - mean_pred: 0.0809 - fmeasure: 0.6576 - precision: 0.7930 - recall: 0.5622 - val_loss: 0.1521 - val_accuracy: 0.6657 - val_mean_pred: 0.0898 - val_fmeasure: 0.5998 - val_precision: 0.7608 - val_recall: 0.4955
Epoch 13/25
504/504 [==============================] - 641s 1s/step - loss: 0.1340 - accuracy: 0.7083 - mean_pred: 0.0810 - fmeasure: 0.6567 - precision: 0.7924 - recall: 0.5611 - val_loss: 0.1520 - val_accuracy: 0.6657 - val_mean_pred: 0.0860 - val_fmeasure: 0.6052 - val_precision: 0.7607 - val_recall: 0.5028
Epoch 14/25
504/504 [==============================] - 645s 1s/step - loss: 0.1332 - accuracy: 0.7105 - mean_pred: 0.0809 - fmeasure: 0.6592 - precision: 0.7936 - recall: 0.5641 - val_loss: 0.1578 - val_accuracy: 0.6461 - val_mean_pred: 0.0872 - val_fmeasure: 0.5881 - val_precision: 0.7364 - val_recall: 0.4899

# Plot training & validation accuracy values
plt.plot(BI_LSTM_model_fit.history['accuracy'])
plt.plot(BI_LSTM_model_fit.history['val_accuracy'])
plt.title('Bil LSTM Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(BI_LSTM_model_fit.history['loss'])
plt.plot(BI_LSTM_model_fit.history['val_loss'])
plt.title('Bil LSTM Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()

#6) GRU
GRU_model = Sequential([
    Embedding(input_dim=max_features, input_length=max_len, output_dim=embedding_dims),
    #So, using this feature will enable the model to look at the context on both the left and the right.
    GRU(64, dropout=0.2, recurrent_dropout=0.2),
    #**batch normalization layer** normalizes the activations of the previous layer at each batch, 
    #i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1. 
    BatchNormalization(),
    Dense(128, activation = 'relu'),
    Dropout(0.2),
    Dense(15, activation = 'sigmoid')
])

GRU_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])

GRU_model.summary()

Model: "sequential_5"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding_5 (Embedding)     (None, 500, 128)          25600000  
                                                                 
 gru (GRU)                   (None, 64)                37248     
                                                                 
 batch_normalization_3 (Batc  (None, 64)               256       
 hNormalization)                                                 
                                                                 
 dense_9 (Dense)             (None, 128)               8320      
                                                                 
 dropout_4 (Dropout)         (None, 128)               0         
                                                                 
 dense_10 (Dense)            (None, 15)                1935      
                                                                 
=================================================================
Total params: 25,647,759
Trainable params: 25,647,631
Non-trainable params: 128
_________________________________________________________________

GRU_model_fit = GRU_model.fit(X_train, y_train, batch_size=batch_size2, epochs=num_epochs, validation_split=val_split, callbacks=[early])

Epoch 1/25
504/504 [==============================] - 897s 2s/step - loss: 0.1621 - accuracy: 0.6464 - mean_pred: 0.0845 - fmeasure: 0.5738 - precision: 0.7349 - recall: 0.4740 - val_loss: 0.1451 - val_accuracy: 0.6680 - val_mean_pred: 0.0808 - val_fmeasure: 0.6061 - val_precision: 0.7585 - val_recall: 0.5050
Epoch 2/25
504/504 [==============================] - 881s 2s/step - loss: 0.1364 - accuracy: 0.7030 - mean_pred: 0.0816 - fmeasure: 0.6506 - precision: 0.7811 - recall: 0.5581 - val_loss: 0.1413 - val_accuracy: 0.6831 - val_mean_pred: 0.0808 - val_fmeasure: 0.6298 - val_precision: 0.7636 - val_recall: 0.5362
Epoch 3/25
504/504 [==============================] - 873s 2s/step - loss: 0.1272 - accuracy: 0.7227 - mean_pred: 0.0814 - fmeasure: 0.6751 - precision: 0.7992 - recall: 0.5848 - val_loss: 0.1424 - val_accuracy: 0.6886 - val_mean_pred: 0.0794 - val_fmeasure: 0.6360 - val_precision: 0.7669 - val_recall: 0.5437
Epoch 4/25
504/504 [==============================] - 867s 2s/step - loss: 0.1219 - accuracy: 0.7330 - mean_pred: 0.0813 - fmeasure: 0.6887 - precision: 0.8096 - recall: 0.5997 - val_loss: 0.1467 - val_accuracy: 0.6812 - val_mean_pred: 0.0783 - val_fmeasure: 0.6368 - val_precision: 0.7490 - val_recall: 0.5542
Epoch 5/25
504/504 [==============================] - 782s 2s/step - loss: 0.1183 - accuracy: 0.7398 - mean_pred: 0.0812 - fmeasure: 0.6981 - precision: 0.8173 - recall: 0.6098 - val_loss: 0.1499 - val_accuracy: 0.6756 - val_mean_pred: 0.0789 - val_fmeasure: 0.6372 - val_precision: 0.7394 - val_recall: 0.5601
Epoch 6/25
504/504 [==============================] - 659s 1s/step - loss: 0.1164 - accuracy: 0.7428 - mean_pred: 0.0812 - fmeasure: 0.7024 - precision: 0.8196 - recall: 0.6151 - val_loss: 0.1498 - val_accuracy: 0.6736 - val_mean_pred: 0.0827 - val_fmeasure: 0.6354 - val_precision: 0.7385 - val_recall: 0.5579

# Plot training & validation accuracy values
plt.plot(GRU_model_fit.history['accuracy'])
plt.plot(GRU_model_fit.history['val_accuracy'])
plt.title('GRU Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(GRU_model_fit.history['loss'])
plt.plot(GRU_model_fit.history['val_loss'])
plt.title('GRU Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()

#7) Bil GRU
BI_GRU_model = Sequential([
    Embedding(input_dim=max_features, input_length=max_len, output_dim=embedding_dims),
    #So, using this feature will enable the model to look at the context on both the left and the right.
    Bidirectional(GRU(units=64, dropout=0.2, recurrent_dropout=0.2)),
    #**batch normalization layer** normalizes the activations of the previous layer at each batch, 
    #i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1. 
    BatchNormalization(),
    Dense(128, activation = 'relu'),
    Dropout(0.2),
    Dense(15, activation = 'sigmoid')
])

BI_GRU_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])

BI_GRU_model.summary()

Model: "sequential_6"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding_6 (Embedding)     (None, 500, 128)          25600000  
                                                                 
 bidirectional_1 (Bidirectio  (None, 128)              74496     
 nal)                                                            
                                                                 
 batch_normalization_4 (Batc  (None, 128)              512       
 hNormalization)                                                 
                                                                 
 dense_11 (Dense)            (None, 128)               16512     
                                                                 
 dropout_5 (Dropout)         (None, 128)               0         
                                                                 
 dense_12 (Dense)            (None, 15)                1935      
                                                                 
=================================================================
Total params: 25,693,455
Trainable params: 25,693,199
Non-trainable params: 256
_________________________________________________________________

BI_GRU_model_fit = GRU_model.fit(X_train, y_train, batch_size=batch_size2, epochs=num_epochs, validation_split=val_split, callbacks=[early])

Epoch 1/25
504/504 [==============================] - 658s 1s/step - loss: 0.1148 - accuracy: 0.7471 - mean_pred: 0.0812 - fmeasure: 0.7076 - precision: 0.8232 - recall: 0.6210 - val_loss: 0.1523 - val_accuracy: 0.6710 - val_mean_pred: 0.0775 - val_fmeasure: 0.6284 - val_precision: 0.7504 - val_recall: 0.5408
Epoch 2/25
504/504 [==============================] - 656s 1s/step - loss: 0.1139 - accuracy: 0.7483 - mean_pred: 0.0811 - fmeasure: 0.7097 - precision: 0.8256 - recall: 0.6229 - val_loss: 0.1585 - val_accuracy: 0.6683 - val_mean_pred: 0.0776 - val_fmeasure: 0.6292 - val_precision: 0.7281 - val_recall: 0.5543
Epoch 3/25
504/504 [==============================] - 2567s 5s/step - loss: 0.1134 - accuracy: 0.7504 - mean_pred: 0.0810 - fmeasure: 0.7107 - precision: 0.8268 - recall: 0.6237 - val_loss: 0.1526 - val_accuracy: 0.6676 - val_mean_pred: 0.0810 - val_fmeasure: 0.6158 - val_precision: 0.7428 - val_recall: 0.5262
Epoch 4/25
504/504 [==============================] - 9721s 19s/step - loss: 0.1122 - accuracy: 0.7512 - mean_pred: 0.0810 - fmeasure: 0.7128 - precision: 0.8283 - recall: 0.6259 - val_loss: 0.1598 - val_accuracy: 0.6561 - val_mean_pred: 0.0783 - val_fmeasure: 0.6041 - val_precision: 0.7331 - val_recall: 0.5141
Epoch 5/25
504/504 [==============================] - 9027s 18s/step - loss: 0.1115 - accuracy: 0.7529 - mean_pred: 0.0809 - fmeasure: 0.7154 - precision: 0.8294 - recall: 0.6294 - val_loss: 0.1624 - val_accuracy: 0.6615 - val_mean_pred: 0.0784 - val_fmeasure: 0.6245 - val_precision: 0.7226 - val_recall: 0.5502

# Plot training & validation accuracy values
plt.plot(BI_GRU_model_fit.history['accuracy'])
plt.plot(BI_GRU_model_fit.history['val_accuracy'])
plt.title('Bil GRU Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(BI_GRU_model_fit.history['loss'])
plt.plot(BI_GRU_model_fit.history['val_loss'])
plt.title('Bil GRU Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()

from chart_studio import plotly
import plotly.offline as py
import plotly.graph_objs as go

#Computing the highest of the evaluation matrics (per model)

trace = go.Table(
    header=dict(values=['Model', 'Loss', 'Accuracy', 'mean_pred', 'F-Measure', 'Precision', 'Recall'],
                line = dict(color='#7D7F80'),
                fill = dict(color='#a1c3d1'),
                align = ['left'] * 5),
    cells=dict(values=[['Neural Network', 'CNN', 'RNNs', 'LSTM', 'BILSTM', 'GRU', 'BIGRU'],
                       [
                        #Loss Evaluation
                        round(np.max(nn_model_fit.history['loss']), 3), round(np.max(CNN_model_fit.history['loss']), 3), round(np.max(RNN_model_fit.history['loss']), 3), 
                        round(np.max(LSTM_model_fit.history['loss']), 3), round(np.max(BI_LSTM_model_fit.history['loss']), 3), round(np.max(GRU_model_fit.history['loss']), 3),
                        round(np.max(BI_GRU_model_fit.history['loss']), 3)],
                        
                        #Accuracy Evaluation
                        [round(np.max(nn_model_fit.history['accuracy']), 3), round(np.max(CNN_model_fit.history['accuracy']), 3), round(np.max(RNN_model_fit.history['accuracy']), 3), 
                        round(np.max(LSTM_model_fit.history['accuracy']), 3), round(np.max(BI_LSTM_model_fit.history['accuracy']), 3), round(np.max(GRU_model_fit.history['accuracy']), 3), 
                        round(np.max(BI_GRU_model_fit.history['accuracy']), 3)],
               
                        #mean_pred Evaluation
                        [round(np.max(nn_model_fit.history['mean_pred']), 3), round(np.max(CNN_model_fit.history['mean_pred']), 3), round(np.max(RNN_model_fit.history['mean_pred']), 3), 
                        round(np.max(LSTM_model_fit.history['mean_pred']), 3), round(np.max(BI_LSTM_model_fit.history['mean_pred']), 3), round(np.max(GRU_model_fit.history['mean_pred']), 3), 
                        round(np.max(BI_GRU_model_fit.history['mean_pred']), 3)],
               
                        #F1-Measure Evaluation
                        [round(np.max(nn_model_fit.history['fmeasure']), 3), round(np.max(CNN_model_fit.history['fmeasure']), 3), round(np.max(RNN_model_fit.history['fmeasure']), 3), 
                        round(np.max(LSTM_model_fit.history['fmeasure']), 3), round(np.max(BI_LSTM_model_fit.history['fmeasure']), 3), round(np.max(GRU_model_fit.history['fmeasure']), 3), 
                        round(np.max(BI_GRU_model_fit.history['fmeasure']), 3)],
               
                        #Precision Evaluation
                        [round(np.max(nn_model_fit.history['precision']), 3), round(np.max(CNN_model_fit.history['precision']), 3), round(np.max(RNN_model_fit.history['precision']), 3), 
                        round(np.max(LSTM_model_fit.history['precision']), 3), round(np.max(BI_LSTM_model_fit.history['precision']), 3), round(np.max(GRU_model_fit.history['precision']), 3), 
                        round(np.max(BI_GRU_model_fit.history['precision']), 3)],
                       
                       
                        #Recall Evaluation
                        [round(np.max(nn_model_fit.history['recall']), 3), round(np.max(CNN_model_fit.history['recall']), 3), round(np.max(RNN_model_fit.history['recall']), 3), 
                        round(np.max(LSTM_model_fit.history['recall']), 3), round(np.max(BI_LSTM_model_fit.history['recall']), 3), round(np.max(GRU_model_fit.history['recall']), 3), 
                        round(np.max(BI_GRU_model_fit.history['recall']), 3)]
                       ],
               line = dict(color='#7D7F80'),
               fill = dict(color='#EDFAFF'),
               align = ['left'] * 5))

layout = dict(width=800, height=400)
data = [trace]
fig = dict(data=data, layout=layout)
py.iplot(data, filename = 'multi-label_with the max of the evaluation matrics (per model) _table')